<!DOCTYPE html>
<html>
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
    <meta name="author" content="Taffy">
    
    <meta name="description" content="纸上得来终觉浅，绝知此事要躬行！">
    
    
    
    
    
    
    <title>爬虫初探 | 标签 | 阿飞</title>
    <link href=”https://taffyblog.github.io“ rel=”prefetch” />

    <link rel="stylesheet" href="/css/bootstrap.min.css">
<link rel="stylesheet" href="/css/aos.css">
<link rel="stylesheet" href="/css/style.css">
    <script src="/js/jquery.min.js"></script>
    <script src="/js/bootstrap.min.js"></script>
    <script src="/js/aos.js"></script>
    <script src="/js/highslide/highslide-full.min.js"></script>
    <link rel="stylesheet" href="/js/highslide/highslide.css">
    <style type="text/css">
        @media (max-width: 768px) {
            body {
                background-color: #f0f0f0;
                background: url('/imgs/xsbg.gif');
                background-attachment: fixed;
            }
        }
    </style>
    
    <!--<script type="text/javascript">
      if (document.images) {
        var avatar = new Image();
        avatar.src = '/imgs/avatar.jpg'
        var previews = 'preview1.jpg,preview2.jpg,preview3.jpg,preview4.jpg'.split(',')
        var previewsPreLoad = []
        for(var i = 0; i < length; i++) {
          previewsPreLoad.push(new Image())
          previewsPreLoad[previewsPreLoad.length - 1].src = '/imgs/preview' + previews[i]
        }
      }
    </script>-->
<link rel="stylesheet" href="/css/prism-vs.css" type="text/css">
<link rel="stylesheet" href="/css/prism-line-numbers.css" type="text/css"></head>
<body>
    <!-- 背景轮播图功能 -->
    <section class="hidden-xs">
    <ul class="cb-slideshow">
        <li><span>天若</span></li>
        <li><span>有情</span></li>
        <li><span>天亦老</span></li>
        <li><span>我为</span></li>
        <li><span>长者</span></li>
        <li><span>续一秒</span></li>
    </ul>
</section>
    <!-- 欧尼酱功能, 谁用谁知道 -->
    
    <div class="gal-menu gal-dropdown">
    <div class="circle" id="gal">
        <div class="ring">
            <a href="https://taffyblog.github.io" class="menuItem" style="left: 50%; top: 15%;">首页</a>
            
            <a class="menuItem" style="left: 80.3109%; top: 32.5%;">下一页</a>
            
            <a href="/archives" class="menuItem" style="left: 80.3109%; top: 67.5%;">归档</a>
            <a href="/about" class="menuItem" style="left: 50%; top: 85%;">关于</a>
            <a href="/message" class="menuItem" style="left: 19.6891%; top: 67.5%;">留言板</a>
            
            <a class="menuItem" style="left: 19.6891%; top: 32.5%;">上一页</a>
            
        </div>
        <audio id="audio" src="/imgs/oni.mp3"></audio>
    </div>
</div>
    
    <header class="navbar navbar-inverse" id="gal-header">
    <div class="container">
        <div class="navbar-header">
            <button type="button" class="navbar-toggle collapsed"
                    data-toggle="collapse" data-target=".bs-navbar-collapse"
                    aria-expanded="false">
                <span class="fa fa-lg fa-reorder"></span>
            </button>
            <a href="https://taffyblog.github.io">
                
                <style>
                    #gal-header .navbar-brand {
                        height: 54px;
                        line-height: 24px;
                        font-size: 28px;
                        opacity: 1;
                        background-color: rgba(0,0,0,0);
                        text-shadow: 0 0 5px #fff,0 0 10px #fff,0 0 15px #fff,0 0 20px #228DFF,0 0 35px #228DFF,0 0 40px #228DFF,0 0 50px #228DFF,0 0 75px #228DFF;
                    }
                </style>
                <!-- 这里使用文字(navbar_text or config.title) -->
                <div class="navbar-brand">小飞</div>
                
            </a>
        </div>
        <div class="collapse navbar-collapse bs-navbar-collapse">
            <ul class="nav navbar-nav" id="menu-gal">
                
                
                <li class="">
                    <a href="/">
                        <i class="fa fa-home"></i>首页
                    </a>
                </li>
                
                
                
                <li class="">
                    <a href="/archives">
                        <i class="fa fa-archive"></i>归档
                    </a>
                </li>
                
                
                
                
                <li class="dropdown">
                    <!-- TODO 添加hover dropdown效果 -->
                    <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                       aria-haspopup="true" aria-expanded="false" data-hover="dropdown">
                        <i class="fa fa-list"></i>分类
                    </a>
                    <ul class="dropdown-menu">
                        
                        
                        <li>
                            <a href="/categories/设计模式/">设计模式</a>
                        </li>
                        
                        <li>
                            <a href="/categories/前端/">前端</a>
                        </li>
                        
                        <li>
                            <a href="/categories/ORM/">ORM</a>
                        </li>
                        
                        
                        <li>
                            <a href="/categories">...</a>
                        </li>
                        
                        
                    </ul>
                </li>
                
                
                
                
                
                <li class="dropdown">
                    <!-- TODO 添加hover dropdown效果 -->
                    <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                       aria-haspopup="true" aria-expanded="false" data-hover="dropdown">
                        <i class="fa fa-tags"></i>标签
                    </a>
                    <ul class="dropdown-menu">
                        
                        
                        <li>
                            <a href="/tags/结构型设计模式/">结构型设计模式</a>
                        </li>
                        
                        <li>
                            <a href="/tags/Javascript/">Javascript</a>
                        </li>
                        
                        <li>
                            <a href="/tags/EF6/">EF6</a>
                        </li>
                        
                        
                        <li>
                            <a href="/tags">...</a>
                        </li>
                        
                        
                    </ul>
                </li>
                
                
                
                
                <li class="">
                    <a href="/links">
                        <i class="fa fa-meanpath"></i>外链
                    </a>
                </li>
                
                
                
                <li class="">
                    <a href="/about">
                        <i class="fa fa-user"></i>关于我
                    </a>
                </li>
                
                
            </ul>
        </div>
    </div>
</header>
    <div id="gal-body">
        <div class="container">
            <div class="row">
                 <!-- link页面不显示侧边栏 -->
                
                    <div class="col-md-8 gal-right" id="mainstay">
                         
<article class="article well article-body" id="article">
    <div class="breadcrumb">
        <i class="fa fa-home"></i>
        <a href="https://taffyblog.github.io">阿飞</a>
        >
        <span>爬虫初探</span>
    </div>
    
    <!-- 大型设备详细文章 -->
    <div class="hidden-xs">
        <div class="title-article">
            <h1>
                <a href="/2017/10/14/Crawler/">爬虫初探</a>
            </h1>
        </div>
        <div class="tag-article">
            
            <span class="label label-gal">
                <i class="fa fa-calendar"></i> 2017-10-14
            </span>
            
        </div>
    </div>
    <!-- 小型设备详细文章 -->
    <div class="visible-xs">
        <center>
            <div class="title-article">
                <h4>
                    <a href="/2017/10/14/Crawler/">爬虫初探</a>
                </h4>
            </div>
            <p>
                <i class="fa fa-calendar"></i> 2017-10-14
            </p>
            <p>
                
                
            </p>
        </center>
    </div>
     <!-- 文章 目录 -->
    <ol class="post-toc"><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#爬虫简介"><span class="post-toc-number">1.</span> <span class="post-toc-text">爬虫简介</span></a></li><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#下载html"><span class="post-toc-number">2.</span> <span class="post-toc-text">下载html</span></a></li><li class="post-toc-item post-toc-level-2"><a class="post-toc-link" href="#HtmlAgilityPack解析Html"><span class="post-toc-number">3.</span> <span class="post-toc-text">HtmlAgilityPack解析Html</span></a></li></ol>
    <div class="content-article">
        <h2 id="爬虫简介"><a href="#爬虫简介" class="headerlink" title="爬虫简介"></a>爬虫简介</h2><p>网络爬虫(又被称为网页蜘蛛，网络机器人，在FOAF社区中间，更经常的称为网页追逐者)，是一种按照一定的规则，自动的抓取万维网信息的程序或者脚本。另外一些不常使用的名字还有蚂蚁，自动索引，模拟程序或者蠕虫。</p>
<h2 id="下载html"><a href="#下载html" class="headerlink" title="下载html"></a>下载html</h2><p>htmlHelper:</p>
<pre class="line-numbers language-java"><code class="language-java">using System<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Collections<span class="token punctuation">.</span>Generic<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>IO<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Linq<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Net<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Text<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Threading<span class="token punctuation">.</span>Tasks<span class="token punctuation">;</span>

namespace Ruanmou<span class="token punctuation">.</span>Crawler
<span class="token punctuation">{</span>
    <span class="token comment" spellcheck="true">/// &lt;summary></span>
    <span class="token comment" spellcheck="true">/// </span>
    <span class="token comment" spellcheck="true">/// &lt;/summary></span>
    <span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">HttpHelper</span>
    <span class="token punctuation">{</span>
        <span class="token keyword">private</span> <span class="token keyword">static</span> Logger logger <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Logger</span><span class="token punctuation">(</span><span class="token function">typeof</span><span class="token punctuation">(</span>HttpHelper<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

        <span class="token comment" spellcheck="true">/// &lt;summary></span>
        <span class="token comment" spellcheck="true">/// 根据url下载内容  之前是GB2312</span>
        <span class="token comment" spellcheck="true">/// &lt;/summary></span>
        <span class="token comment" spellcheck="true">/// &lt;param name="url">&lt;/param></span>
        <span class="token comment" spellcheck="true">/// &lt;returns>&lt;/returns></span>
        <span class="token keyword">public</span> <span class="token keyword">static</span> string <span class="token function">DownloadUrl</span><span class="token punctuation">(</span>string url<span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            <span class="token keyword">return</span> <span class="token function">DownloadHtml</span><span class="token punctuation">(</span>url<span class="token punctuation">,</span> Encoding<span class="token punctuation">.</span>UTF8<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">}</span>

        <span class="token comment" spellcheck="true">/// &lt;summary></span>
        <span class="token comment" spellcheck="true">/// 下载html</span>
        <span class="token comment" spellcheck="true">/// HttpWebRequest功能比较丰富，WebClient使用比较简单</span>
        <span class="token comment" spellcheck="true">/// &lt;/summary></span>
        <span class="token comment" spellcheck="true">/// &lt;param name="url">&lt;/param></span>
        <span class="token comment" spellcheck="true">/// &lt;returns>&lt;/returns></span>
        <span class="token keyword">public</span> <span class="token keyword">static</span> string <span class="token function">DownloadHtml</span><span class="token punctuation">(</span>string url<span class="token punctuation">,</span> Encoding encode<span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            string html <span class="token operator">=</span> string<span class="token punctuation">.</span>Empty<span class="token punctuation">;</span>
            <span class="token keyword">try</span>
            <span class="token punctuation">{</span>
                HttpWebRequest request <span class="token operator">=</span> HttpWebRequest<span class="token punctuation">.</span><span class="token function">Create</span><span class="token punctuation">(</span>url<span class="token punctuation">)</span> as HttpWebRequest<span class="token punctuation">;</span><span class="token comment" spellcheck="true">//模拟请求</span>
                request<span class="token punctuation">.</span>Timeout <span class="token operator">=</span> <span class="token number">30</span> <span class="token operator">*</span> <span class="token number">1000</span><span class="token punctuation">;</span><span class="token comment" spellcheck="true">//设置30s的超时</span>
                request<span class="token punctuation">.</span>UserAgent <span class="token operator">=</span> <span class="token string">"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"</span><span class="token punctuation">;</span>
                request<span class="token punctuation">.</span>ContentType <span class="token operator">=</span> <span class="token string">"text/html; charset=utf-8"</span><span class="token punctuation">;</span>

                <span class="token function">using</span> <span class="token punctuation">(</span>HttpWebResponse response <span class="token operator">=</span> request<span class="token punctuation">.</span><span class="token function">GetResponse</span><span class="token punctuation">(</span><span class="token punctuation">)</span> as HttpWebResponse<span class="token punctuation">)</span><span class="token comment" spellcheck="true">//发起请求</span>
                <span class="token punctuation">{</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>response<span class="token punctuation">.</span>StatusCode <span class="token operator">!=</span> HttpStatusCode<span class="token punctuation">.</span>OK<span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        logger<span class="token punctuation">.</span><span class="token function">Warn</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"抓取{0}地址返回失败,response.StatusCode为{1}"</span><span class="token punctuation">,</span> url<span class="token punctuation">,</span> response<span class="token punctuation">.</span>StatusCode<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    <span class="token keyword">else</span>
                    <span class="token punctuation">{</span>
                        <span class="token keyword">try</span>
                        <span class="token punctuation">{</span>
                            StreamReader sr <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">StreamReader</span><span class="token punctuation">(</span>response<span class="token punctuation">.</span><span class="token function">GetResponseStream</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">,</span> encode<span class="token punctuation">)</span><span class="token punctuation">;</span>
                            html <span class="token operator">=</span> sr<span class="token punctuation">.</span><span class="token function">ReadToEnd</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span><span class="token comment" spellcheck="true">//读取数据</span>
                            sr<span class="token punctuation">.</span><span class="token function">Close</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                        <span class="token punctuation">}</span>
                        <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span>
                        <span class="token punctuation">{</span>
                            logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span>$<span class="token string">"DownloadHtml抓取{url}失败"</span><span class="token punctuation">)</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
                            html <span class="token operator">=</span> null<span class="token punctuation">;</span>
                        <span class="token punctuation">}</span>
                    <span class="token punctuation">}</span>
                <span class="token punctuation">}</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">System<span class="token punctuation">.</span>Net<span class="token punctuation">.</span>WebException</span> ex<span class="token punctuation">)</span>
            <span class="token punctuation">{</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>ex<span class="token punctuation">.</span>Message<span class="token punctuation">.</span><span class="token function">Equals</span><span class="token punctuation">(</span><span class="token string">"远程服务器返回错误: (306)。"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span><span class="token string">"远程服务器返回错误: (306)。"</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
                    html <span class="token operator">=</span> null<span class="token punctuation">;</span>
                <span class="token punctuation">}</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span>
            <span class="token punctuation">{</span>
                logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"DownloadHtml抓取{0}出现异常"</span><span class="token punctuation">,</span> url<span class="token punctuation">)</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
                html <span class="token operator">=</span> null<span class="token punctuation">;</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">return</span> html<span class="token punctuation">;</span>
        <span class="token punctuation">}</span>
    <span class="token punctuation">}</span>
<span class="token punctuation">}</span>

<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<pre class="line-numbers language-java"><code class="language-java">

<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<h2 id="HtmlAgilityPack解析Html"><a href="#HtmlAgilityPack解析Html" class="headerlink" title="HtmlAgilityPack解析Html"></a>HtmlAgilityPack解析Html</h2><p>商品抓取</p>
<pre class="line-numbers language-java"><code class="language-java">using HtmlAgilityPack<span class="token punctuation">;</span>
using System<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Collections<span class="token punctuation">.</span>Generic<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>IO<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Linq<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Net<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Text<span class="token punctuation">;</span>
using System<span class="token punctuation">.</span>Threading<span class="token punctuation">.</span>Tasks<span class="token punctuation">;</span>
using Newtonsoft<span class="token punctuation">.</span>Json<span class="token punctuation">;</span>
using Ruanmou<span class="token punctuation">.</span>Crawler<span class="token punctuation">.</span>DataService<span class="token punctuation">;</span>

using Ruanmou<span class="token punctuation">.</span>Crawler<span class="token punctuation">.</span>Model<span class="token punctuation">;</span>

namespace Ruanmou<span class="token punctuation">.</span>Crawler
<span class="token punctuation">{</span>
    <span class="token comment" spellcheck="true">/// &lt;summary></span>
    <span class="token comment" spellcheck="true">/// 商品抓取</span>
    <span class="token comment" spellcheck="true">/// http://www.w3school.com.cn/xpath/index.asp XPATH语法</span>
    <span class="token comment" spellcheck="true">/// &lt;/summary></span>
    <span class="token keyword">public</span> <span class="token keyword">class</span> <span class="token class-name">CommoditySearch</span> <span class="token operator">:</span> ISearch
    <span class="token punctuation">{</span>
        <span class="token keyword">private</span> Logger logger <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Logger</span><span class="token punctuation">(</span><span class="token function">typeof</span><span class="token punctuation">(</span>CommoditySearch<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">private</span> WarnRepository warnRepository <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">WarnRepository</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">private</span> CommodityRepository commodityRepository <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">CommodityRepository</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">private</span> Category category <span class="token operator">=</span> null<span class="token punctuation">;</span>

        <span class="token keyword">public</span> <span class="token function">CommoditySearch</span><span class="token punctuation">(</span>Category _category<span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            category <span class="token operator">=</span> _category<span class="token punctuation">;</span>
        <span class="token punctuation">}</span>

        <span class="token keyword">public</span> <span class="token keyword">void</span> <span class="token function">Crawler</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            <span class="token keyword">try</span>
            <span class="token punctuation">{</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">IsNullOrEmpty</span><span class="token punctuation">(</span>category<span class="token punctuation">.</span>Url<span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    warnRepository<span class="token punctuation">.</span><span class="token function">SaveWarn</span><span class="token punctuation">(</span>category<span class="token punctuation">,</span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"Url为空,Name={0} Level={1} Url={2}"</span><span class="token punctuation">,</span> category<span class="token punctuation">.</span>Name<span class="token punctuation">,</span> category<span class="token punctuation">.</span>CategoryLevel<span class="token punctuation">,</span> category<span class="token punctuation">.</span>Url<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">return</span><span class="token punctuation">;</span>
                <span class="token punctuation">}</span>
                string html <span class="token operator">=</span> HttpHelper<span class="token punctuation">.</span><span class="token function">DownloadUrl</span><span class="token punctuation">(</span>category<span class="token punctuation">.</span>Url<span class="token punctuation">)</span><span class="token punctuation">;</span><span class="token comment" spellcheck="true">//下载html</span>

                HtmlDocument doc <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">HtmlDocument</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                doc<span class="token punctuation">.</span><span class="token function">LoadHtml</span><span class="token punctuation">(</span>html<span class="token punctuation">)</span><span class="token punctuation">;</span><span class="token comment" spellcheck="true">//加载html</span>
                string pageNumberPath <span class="token operator">=</span> @<span class="token string">"//*[@id='J_topPage']/span/i"</span><span class="token punctuation">;</span>
                HtmlNode pageNumberNode <span class="token operator">=</span> doc<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectSingleNode</span><span class="token punctuation">(</span>pageNumberPath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>pageNumberNode <span class="token operator">!=</span> null<span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    string sNumber <span class="token operator">=</span> pageNumberNode<span class="token punctuation">.</span>InnerText<span class="token punctuation">;</span>
                    <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">;</span> i <span class="token operator">&lt;</span> <span class="token keyword">int</span><span class="token punctuation">.</span><span class="token function">Parse</span><span class="token punctuation">(</span>sNumber<span class="token punctuation">)</span> <span class="token operator">+</span> <span class="token number">1</span><span class="token punctuation">;</span> i<span class="token operator">++</span><span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        string pageUrl <span class="token operator">=</span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"{0}&amp;page={1}"</span><span class="token punctuation">,</span> category<span class="token punctuation">.</span>Url<span class="token punctuation">,</span> i<span class="token punctuation">)</span><span class="token punctuation">;</span>
                        <span class="token keyword">try</span>
                        <span class="token punctuation">{</span>
                            List<span class="token operator">&lt;</span>Commodity<span class="token operator">></span> commodityList <span class="token operator">=</span> <span class="token function">GetCommodityList</span><span class="token punctuation">(</span>category<span class="token punctuation">,</span> pageUrl<span class="token punctuation">.</span><span class="token function">Replace</span><span class="token punctuation">(</span><span class="token string">"&amp;page=1&amp;"</span><span class="token punctuation">,</span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"&amp;page={0}&amp;"</span><span class="token punctuation">,</span> i<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                            <span class="token comment" spellcheck="true">//commodityRepository.SaveList(commodityList);</span>
                        <span class="token punctuation">}</span>
                        <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span><span class="token comment" spellcheck="true">//保证一页的错误不影响另外一页</span>
                        <span class="token punctuation">{</span>
                            logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span><span class="token string">"Crawler的commodityRepository.SaveList(commodityList)出现异常"</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
                        <span class="token punctuation">}</span>
                    <span class="token punctuation">}</span>
                <span class="token punctuation">}</span>




                <span class="token comment" spellcheck="true">//string fristPath = "//*[@id='J_bottomPage']/span[1]/a";</span>
                <span class="token comment" spellcheck="true">//HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(fristPath);//xPath分析</span>
                <span class="token comment" spellcheck="true">//if (noneNodeList == null)</span>
                <span class="token comment" spellcheck="true">//{</span>
                <span class="token comment" spellcheck="true">//    warnRepository.SaveWarn(category, string.Format("分页数据为空,Name={0} Level={1} Url={2}", category.Name, category.CategoryLevel, category.Url));</span>
                <span class="token comment" spellcheck="true">//    return;</span>
                <span class="token comment" spellcheck="true">//}</span>

                <span class="token comment" spellcheck="true">//string pageUrl = null;</span>
                <span class="token comment" spellcheck="true">//foreach (var node in noneNodeList)</span>
                <span class="token comment" spellcheck="true">//{</span>
                <span class="token comment" spellcheck="true">//    string sNum = node.InnerHtml;</span>
                <span class="token comment" spellcheck="true">//    if (sNum.Equals("1"))</span>
                <span class="token comment" spellcheck="true">//    {</span>
                <span class="token comment" spellcheck="true">//        pageUrl = node.Attributes["href"].Value.Replace("&amp;amp;", "&amp;");</span>
                <span class="token comment" spellcheck="true">//        if (!pageUrl.StartsWith("http://"))</span>
                <span class="token comment" spellcheck="true">//            pageUrl = string.Format("http://list.jd.com{0}", pageUrl);</span>
                <span class="token comment" spellcheck="true">//        break;</span>
                <span class="token comment" spellcheck="true">//    }</span>
                <span class="token comment" spellcheck="true">//}</span>
                <span class="token comment" spellcheck="true">//string sMaxPageNumPath = "//*[@id='J_bottomPage']/span[2]/em[1]/b";</span>
                <span class="token comment" spellcheck="true">//HtmlNode sMaxPageNumPathNode = doc.DocumentNode.SelectSingleNode(sMaxPageNumPath);</span>
                <span class="token comment" spellcheck="true">//string sMaxPageNum = sMaxPageNumPathNode.InnerHtml;</span>
                <span class="token comment" spellcheck="true">//for (int i = 1; i &lt; int.Parse(sMaxPageNum) + 1; i++)</span>
                <span class="token comment" spellcheck="true">//{</span>
                <span class="token comment" spellcheck="true">//    try</span>
                <span class="token comment" spellcheck="true">//    {</span>
                <span class="token comment" spellcheck="true">//        List&lt;Commodity> commodityList = GetCommodityList(category, pageUrl.Replace("&amp;page=1&amp;", string.Format("&amp;page={0}&amp;", i)));</span>
                <span class="token comment" spellcheck="true">//        commodityRepository.SaveList(commodityList);</span>
                <span class="token comment" spellcheck="true">//    }</span>
                <span class="token comment" spellcheck="true">//    catch (Exception ex)//保证一页的错误不影响另外一页</span>
                <span class="token comment" spellcheck="true">//    {</span>
                <span class="token comment" spellcheck="true">//        logger.Error("Crawler的commodityRepository.SaveList(commodityList)出现异常", ex);</span>
                <span class="token comment" spellcheck="true">//    }</span>
                <span class="token comment" spellcheck="true">//}</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span>
            <span class="token punctuation">{</span>
                logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span><span class="token string">"CrawlerMuti出现异常"</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
                warnRepository<span class="token punctuation">.</span><span class="token function">SaveWarn</span><span class="token punctuation">(</span>category<span class="token punctuation">,</span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"出现异常,Name={0} Level={1} Url={2}"</span><span class="token punctuation">,</span> category<span class="token punctuation">.</span>Name<span class="token punctuation">,</span> category<span class="token punctuation">.</span>CategoryLevel<span class="token punctuation">,</span> category<span class="token punctuation">.</span>Url<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">}</span>
        <span class="token punctuation">}</span>

        <span class="token keyword">private</span> List<span class="token operator">&lt;</span>Commodity<span class="token operator">></span> <span class="token function">GetCommodityList</span><span class="token punctuation">(</span>Category category<span class="token punctuation">,</span> string url<span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            string html <span class="token operator">=</span> HttpHelper<span class="token punctuation">.</span><span class="token function">DownloadUrl</span><span class="token punctuation">(</span>url<span class="token punctuation">)</span><span class="token punctuation">;</span>
            List<span class="token operator">&lt;</span>Commodity<span class="token operator">></span> commodityList <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">List</span><span class="token operator">&lt;</span>Commodity<span class="token operator">></span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token keyword">try</span>
            <span class="token punctuation">{</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">IsNullOrEmpty</span><span class="token punctuation">(</span>html<span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token keyword">return</span> commodityList<span class="token punctuation">;</span>
                HtmlDocument doc <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">HtmlDocument</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                doc<span class="token punctuation">.</span><span class="token function">LoadHtml</span><span class="token punctuation">(</span>html<span class="token punctuation">)</span><span class="token punctuation">;</span>
                string liPath <span class="token operator">=</span> <span class="token string">"//*[@id='plist']/ul/li"</span><span class="token punctuation">;</span>
                HtmlNodeCollection noneNodeList <span class="token operator">=</span> doc<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectNodes</span><span class="token punctuation">(</span>liPath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>noneNodeList <span class="token operator">==</span> null <span class="token operator">||</span> noneNodeList<span class="token punctuation">.</span>Count <span class="token operator">==</span> <span class="token number">0</span><span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    warnRepository<span class="token punctuation">.</span><span class="token function">SaveWarn</span><span class="token punctuation">(</span>category<span class="token punctuation">,</span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"GetCommodityList商品数据为空,Name={0} Level={1} category.Url={2} url={3}"</span><span class="token punctuation">,</span> category<span class="token punctuation">.</span>Name<span class="token punctuation">,</span> category<span class="token punctuation">.</span>CategoryLevel<span class="token punctuation">,</span> category<span class="token punctuation">.</span>Url<span class="token punctuation">,</span> url<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">return</span> commodityList<span class="token punctuation">;</span>
                <span class="token punctuation">}</span>
                <span class="token function">foreach</span> <span class="token punctuation">(</span>var node in noneNodeList<span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    HtmlDocument docChild <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">HtmlDocument</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                    docChild<span class="token punctuation">.</span><span class="token function">LoadHtml</span><span class="token punctuation">(</span>node<span class="token punctuation">.</span>OuterHtml<span class="token punctuation">)</span><span class="token punctuation">;</span>

                    Commodity commodity <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Commodity</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        CategoryId <span class="token operator">=</span> category<span class="token punctuation">.</span>Id
                    <span class="token punctuation">}</span><span class="token punctuation">;</span>

                    string urlPath <span class="token operator">=</span> <span class="token string">"//*[@class='p-name']/a"</span><span class="token punctuation">;</span>
                    HtmlNode urlNode <span class="token operator">=</span> docChild<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectSingleNode</span><span class="token punctuation">(</span>urlPath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>urlNode <span class="token operator">==</span> null<span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    commodity<span class="token punctuation">.</span>Url <span class="token operator">=</span> urlNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">[</span><span class="token string">"href"</span><span class="token punctuation">]</span><span class="token punctuation">.</span>Value<span class="token punctuation">;</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>commodity<span class="token punctuation">.</span>Url<span class="token punctuation">.</span><span class="token function">StartsWith</span><span class="token punctuation">(</span><span class="token string">"http:"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                        commodity<span class="token punctuation">.</span>Url <span class="token operator">=</span> <span class="token string">"http:"</span> <span class="token operator">+</span> commodity<span class="token punctuation">.</span>Url<span class="token punctuation">;</span>

                    string sId <span class="token operator">=</span> Path<span class="token punctuation">.</span><span class="token function">GetFileName</span><span class="token punctuation">(</span>commodity<span class="token punctuation">.</span>Url<span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">Replace</span><span class="token punctuation">(</span><span class="token string">".html"</span><span class="token punctuation">,</span> <span class="token string">""</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                    commodity<span class="token punctuation">.</span>ProductId <span class="token operator">=</span> <span class="token keyword">long</span><span class="token punctuation">.</span><span class="token function">Parse</span><span class="token punctuation">(</span>sId<span class="token punctuation">)</span><span class="token punctuation">;</span>

                    <span class="token comment" spellcheck="true">//*[@id="plist"]/ul/li[1]/div/div[3]/a/em</span>
                    string titlePath <span class="token operator">=</span> <span class="token string">"//*[@class='p-name']/a/em"</span><span class="token punctuation">;</span>
                    HtmlNode titleNode <span class="token operator">=</span> docChild<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectSingleNode</span><span class="token punctuation">(</span>titlePath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>titleNode <span class="token operator">==</span> null<span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        <span class="token comment" spellcheck="true">//Log.Error(titlePath);</span>
                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    commodity<span class="token punctuation">.</span>Title <span class="token operator">=</span> titleNode<span class="token punctuation">.</span>InnerText<span class="token punctuation">;</span>

                    string iamgePath <span class="token operator">=</span> <span class="token string">"//*[@class='p-img']/a/img"</span><span class="token punctuation">;</span>
                    HtmlNode imageNode <span class="token operator">=</span> docChild<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectSingleNode</span><span class="token punctuation">(</span>iamgePath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>imageNode <span class="token operator">==</span> null<span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    <span class="token comment" spellcheck="true">//前后不一</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">.</span><span class="token function">Contains</span><span class="token punctuation">(</span><span class="token string">"src"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                        commodity<span class="token punctuation">.</span>ImageUrl <span class="token operator">=</span> imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">[</span><span class="token string">"src"</span><span class="token punctuation">]</span><span class="token punctuation">.</span>Value<span class="token punctuation">;</span>
                    <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">.</span><span class="token function">Contains</span><span class="token punctuation">(</span><span class="token string">"original"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                        commodity<span class="token punctuation">.</span>ImageUrl <span class="token operator">=</span> imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">[</span><span class="token string">"original"</span><span class="token punctuation">]</span><span class="token punctuation">.</span>Value<span class="token punctuation">;</span>
                    <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">.</span><span class="token function">Contains</span><span class="token punctuation">(</span><span class="token string">"data-lazy-img"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                        commodity<span class="token punctuation">.</span>ImageUrl <span class="token operator">=</span> imageNode<span class="token punctuation">.</span>Attributes<span class="token punctuation">[</span><span class="token string">"data-lazy-img"</span><span class="token punctuation">]</span><span class="token punctuation">.</span>Value<span class="token punctuation">;</span>
                    <span class="token keyword">else</span>
                    <span class="token punctuation">{</span>
                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>commodity<span class="token punctuation">.</span>ImageUrl<span class="token punctuation">.</span><span class="token function">StartsWith</span><span class="token punctuation">(</span><span class="token string">"http:"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
                        commodity<span class="token punctuation">.</span>ImageUrl <span class="token operator">=</span> <span class="token string">"http:"</span> <span class="token operator">+</span> commodity<span class="token punctuation">.</span>ImageUrl<span class="token punctuation">;</span>

                    string pricePath <span class="token operator">=</span> <span class="token string">"//*[@class='p-price']/strong/i"</span><span class="token punctuation">;</span>
                    HtmlNode priceNode <span class="token operator">=</span> docChild<span class="token punctuation">.</span>DocumentNode<span class="token punctuation">.</span><span class="token function">SelectSingleNode</span><span class="token punctuation">(</span>pricePath<span class="token punctuation">)</span><span class="token punctuation">;</span>
                    <span class="token keyword">if</span> <span class="token punctuation">(</span>priceNode <span class="token operator">==</span> null<span class="token punctuation">)</span>
                    <span class="token punctuation">{</span>
                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
                    <span class="token punctuation">}</span>
                    <span class="token keyword">else</span>
                    <span class="token punctuation">{</span>
                    <span class="token punctuation">}</span>
                    commodityList<span class="token punctuation">.</span><span class="token function">Add</span><span class="token punctuation">(</span>commodity<span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token punctuation">}</span>
                Console<span class="token punctuation">.</span><span class="token function">WriteLine</span><span class="token punctuation">(</span><span class="token string">"{0}一共获取了{1}条数据"</span><span class="token punctuation">,</span> url<span class="token punctuation">,</span> commodityList<span class="token punctuation">.</span>Count<span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span>
            <span class="token punctuation">{</span>
                logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"GetCommodityList出现异常,url={0}"</span><span class="token punctuation">,</span> url<span class="token punctuation">)</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">return</span> <span class="token function">GetCommodityPrice</span><span class="token punctuation">(</span>category<span class="token punctuation">,</span> commodityList<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token punctuation">}</span>

        <span class="token comment" spellcheck="true">/// &lt;summary></span>
        <span class="token comment" spellcheck="true">/// 获取商品价格</span>
        <span class="token comment" spellcheck="true">/// &lt;/summary></span>
        <span class="token comment" spellcheck="true">/// &lt;param name="commodityList">&lt;/param></span>
        <span class="token comment" spellcheck="true">/// &lt;returns>&lt;/returns></span>
        <span class="token keyword">private</span> List<span class="token operator">&lt;</span>Commodity<span class="token operator">></span> <span class="token function">GetCommodityPrice</span><span class="token punctuation">(</span>Category category<span class="token punctuation">,</span> List<span class="token operator">&lt;</span>Commodity<span class="token operator">></span> commodityList<span class="token punctuation">)</span>
        <span class="token punctuation">{</span>
            <span class="token keyword">try</span>
            <span class="token punctuation">{</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>commodityList <span class="token operator">==</span> null <span class="token operator">||</span> commodityList<span class="token punctuation">.</span><span class="token function">Count</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">==</span> <span class="token number">0</span><span class="token punctuation">)</span>
                    <span class="token keyword">return</span> commodityList<span class="token punctuation">;</span>

                StringBuilder sb <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">StringBuilder</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token comment" spellcheck="true">//sb.Append(@"http://p.3.cn/prices/mgets?my=list_price&amp;type=1&amp;area=1_72_4137&amp;skuIds=");</span>
                <span class="token comment" spellcheck="true">//sb.Append(string.Join("%2C", commodityList.Select(c => string.Format("J_{0}", c.ProductId))));</span>
                <span class="token comment" spellcheck="true">//</span>
                sb<span class="token punctuation">.</span><span class="token function">AppendFormat</span><span class="token punctuation">(</span><span class="token string">"http://p.3.cn/prices/mgets?callback=jQuery1069298&amp;type=1&amp;area=1_72_4137_0&amp;skuIds={0}&amp;pdbp=0&amp;pdtk=&amp;pdpin=&amp;pduid=1945966343&amp;_=1469022843655"</span><span class="token punctuation">,</span> string<span class="token punctuation">.</span><span class="token function">Join</span><span class="token punctuation">(</span><span class="token string">"%2C"</span><span class="token punctuation">,</span> commodityList<span class="token punctuation">.</span><span class="token function">Select</span><span class="token punctuation">(</span>c <span class="token operator">=</span><span class="token operator">></span> string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"J_{0}"</span><span class="token punctuation">,</span> c<span class="token punctuation">.</span>ProductId<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                string html <span class="token operator">=</span> HttpHelper<span class="token punctuation">.</span><span class="token function">DownloadUrl</span><span class="token punctuation">(</span>sb<span class="token punctuation">.</span><span class="token function">ToString</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token keyword">if</span> <span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">IsNullOrWhiteSpace</span><span class="token punctuation">(</span>html<span class="token punctuation">)</span><span class="token punctuation">)</span>
                <span class="token punctuation">{</span>
                    logger<span class="token punctuation">.</span><span class="token function">Warn</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"获取url={0}时获取的html为空"</span><span class="token punctuation">,</span> sb<span class="token punctuation">.</span><span class="token function">ToString</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token punctuation">}</span>
                html <span class="token operator">=</span> html<span class="token punctuation">.</span><span class="token function">Substring</span><span class="token punctuation">(</span>html<span class="token punctuation">.</span><span class="token function">IndexOf</span><span class="token punctuation">(</span><span class="token string">"("</span><span class="token punctuation">)</span> <span class="token operator">+</span> <span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                html <span class="token operator">=</span> html<span class="token punctuation">.</span><span class="token function">Substring</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">,</span> html<span class="token punctuation">.</span><span class="token function">LastIndexOf</span><span class="token punctuation">(</span><span class="token string">")"</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
                List<span class="token operator">&lt;</span>CommodityPrice<span class="token operator">></span> priceList <span class="token operator">=</span> JsonConvert<span class="token punctuation">.</span>DeserializeObject<span class="token operator">&lt;</span>List<span class="token operator">&lt;</span>CommodityPrice<span class="token operator">>></span><span class="token punctuation">(</span>html<span class="token punctuation">)</span><span class="token punctuation">;</span>
                commodityList<span class="token punctuation">.</span><span class="token function">ForEach</span><span class="token punctuation">(</span>c <span class="token operator">=</span><span class="token operator">></span> c<span class="token punctuation">.</span>Price <span class="token operator">=</span> priceList<span class="token punctuation">.</span><span class="token function">FirstOrDefault</span><span class="token punctuation">(</span>p <span class="token operator">=</span><span class="token operator">></span> p<span class="token punctuation">.</span>id<span class="token punctuation">.</span><span class="token function">Equals</span><span class="token punctuation">(</span>string<span class="token punctuation">.</span><span class="token function">Format</span><span class="token punctuation">(</span><span class="token string">"J_{0}"</span><span class="token punctuation">,</span> c<span class="token punctuation">.</span>ProductId<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">.</span>p<span class="token punctuation">)</span><span class="token punctuation">;</span>
                <span class="token comment" spellcheck="true">//commodityList.ForEach(c => Console.WriteLine(" Title={0}  ImageUrl={1} Url={2} Price={3} Id={4}", c.Title, c.ImageUrl, c.Url, c.Price, c.Id));</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> ex<span class="token punctuation">)</span>
            <span class="token punctuation">{</span>
                logger<span class="token punctuation">.</span><span class="token function">Error</span><span class="token punctuation">(</span><span class="token string">"GetCommodityPrice出现异常"</span><span class="token punctuation">,</span> ex<span class="token punctuation">)</span><span class="token punctuation">;</span>
            <span class="token punctuation">}</span>
            <span class="token keyword">return</span> commodityList<span class="token punctuation">;</span>
        <span class="token punctuation">}</span>
    <span class="token punctuation">}</span>
<span class="token punctuation">}</span>


<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p><a href="https://gitee.com/taffy88/Ruanmou.Crawler" target="_blank" rel="noopener">完整项目地址</a></p>

    </div>
</article>


<div id="comments-template"></div>
<link rel="stylesheet" href="https://imsun.github.io/gitment/style/default.css">
<script>
	if(!window.commentConfig) {
      window.commentConfig = {}
      window.commentConfig.title = '爬虫初探'
    }
</script>

                    </div>
                    <aside class="col-md-4 gal-left" id="sidebar">
    <!-- 此为sidebar的搜索框, 非搜索结果页面 -->
<aside id="sidebar-search">
    <div class="search hidden-xs" data-aos="fade-up" data-aos-duration="2000">
        <form class="form-inline clearfix" id="search-form" method="get"
              action="/search/index.html">
            <input type="text" name="s" class="form-control" id="searchInput" placeholder="搜索文章~">
            <button class="btn btn-danger btn-gal" type="submit">
                <i class="fa fa-search"></i>
            </button>
        </form>
    </div>
</aside>
    <aside id="sidebar-author">
    <div class="panel panel-gal" data-aos="flip-right" data-aos-duration="3000">
        <div class="panel-heading" style="text-align: center">
            <i class="fa fa-quote-left"></i>
            Taffy
            <i class="fa fa-quote-right"></i>
        </div>
        <div class="author-panel text-center">
            <img src="/imgs/avatar.jpg" width="140" height="140"
                 alt="个人头像" class="author-image">
            <p class="author-description"><p>纸上得来终觉浅，绝知此事要躬行！</p>
</p>
        </div>
    </div>
</aside>
    
    <aside id="sidebar-recent_comments">
    <div class="panel panel-gal recent hidden-xs" data-aos="fade-up" data-aos-duration="2000">
        <div class="panel-heading">
            <i class="fa fa-comments"></i>
            最新评论
            <i class="fa fa-times-circle panel-remove"></i>
            <i class="fa fa-chevron-circle-up panel-toggle"></i>
        </div>
        <ul class="list-group list-group-flush"></ul>
    </div>
</aside>
    
    <!-- 要配置好leancloud才能开启此小工具 -->
    
    
    <aside id="sidebar-recent_posts">
    <div class="panel panel-gal recent hidden-xs" data-aos="fade-up" data-aos-duration="2000">
        <div class="panel-heading">
            <i class="fa fa-refresh"></i>
            近期文章
            <i class="fa fa-times-circle panel-remove"></i>
            <i class="fa fa-chevron-circle-up panel-toggle"></i>
        </div>
        <ul class="list-group list-group-flush">
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/08/14/vueJsRoute/">VueJs路由入门</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/31/vueJsCompoentData/">VueJs组件间的数据交换</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/30/vueJsComponentSendData/">VueJs组件插槽、动态组件、缓存组件</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/26/vueJswebpack/">VueJs通过webpack启服务运行VUE模块文件</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/25/vuejscmd/">VueJs自定义指令和组件</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/25/vueAttr/">VueJs实例属性</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/20/vueLifeCycle/">VueJs生命周期、计算属性和侦听器</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/17/vueJsFilter/">VueJs过滤器</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/16/vueJsEvent/">VueJs事件操作</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/12/vuejstemplete/">VueJs表单数据操作</a>
                </span>
            </li>
            
        </ul>
    </div>
</aside>
    
    
    <aside id="sidebar-rand_posts">
    <div class="panel panel-gal recent hidden-xs" data-aos="fade-up" data-aos-duration="2000">
        <div class="panel-heading">
            <i class="fa fa-refresh"></i>
            随机文章
            <i class="fa fa-times-circle panel-remove"></i>
            <i class="fa fa-chevron-circle-up panel-toggle"></i>
        </div>
        <ul class="list-group list-group-flush">
            
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/06/22/DestructuringAssignment/">Javascript解构赋值</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/08/19/Ioc/">控制反转/依赖注入（Unity容器使用）</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/05/21/factorymethod/">设计模式(二)--工厂方法模式</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/05/13/varAndletDifference/">var和let的作用域区别</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2018/07/12/vuejstemplete/">VueJs表单数据操作</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/05/12/aspdownbigfile/">ASP.NET断点续传大文件</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/07/02/encryption/">加密和解密</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/05/18/fanxing/">C#中的泛型</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/09/17/httpAndWebSocket/">HTTP协议与WebSocket</a>
                </span>
            </li>
            
            <li class="list-group-item">
                <span class="post-title">
                    <a href="/2017/05/20/serializable/">对象序列化</a>
                </span>
            </li>
            
        </ul>
    </div>
</aside>
    
    
    <aside id="gal-sets">
        <div class="panel panel-gal hidden-xs" data-aos="fade-up" data-aos-duration="2000">
            <ul class="nav nav-pills pills-gal">
                <li class="">
                    <a href="/2017/10/14/Crawler/index.html#sidebar-tags" data-toggle="tab" id="tags-tab">热门标签</a>
                </li>
                <li class="">
                    <a href="/2017/10/14/Crawler/index.html#sidebar-friend-links" data-toggle="tab" id="friend-links-tab">友情链接</a>
                </li>
                <li class="">
                    <a href="/2017/10/14/Crawler/index.html#sidebar-links" data-toggle="tab" id="links-tab">个人链接</a>
                </li>
            </ul>
            <div class="tab-content">
                <div class="cloud-tags tab-pane nav bs-sidenav fade" id="sidebar-tags">
    
    <a href="/tags/结构型设计模式/" style="font-size: 19.904618632749568px;" class="tag-cloud-link">结构型设计模式</a>
    
    <a href="/tags/Javascript/" style="font-size: 17.487695806411562px;" class="tag-cloud-link">Javascript</a>
    
    <a href="/tags/EF6/" style="font-size: 9.167839620131762px;" class="tag-cloud-link">EF6</a>
    
    <a href="/tags/创建型设计模式/" style="font-size: 17.007522080431624px;" class="tag-cloud-link">创建型设计模式</a>
    
    <a href="/tags/程序集/" style="font-size: 8.957318408056992px;" class="tag-cloud-link">程序集</a>
    
    <a href="/tags/CSS/" style="font-size: 17.73660681558648px;" class="tag-cloud-link">CSS</a>
    
    <a href="/tags/javascript/" style="font-size: 11.262571194125083px;" class="tag-cloud-link">javascript</a>
    
    <a href="/tags/正则表达式/" style="font-size: 9.23523859173396px;" class="tag-cloud-link">正则表达式</a>
    
    <a href="/tags/WinForm/" style="font-size: 11.3825809561922px;" class="tag-cloud-link">WinForm</a>
    
    <a href="/tags/多线程/" style="font-size: 9.433185012515px;" class="tag-cloud-link">多线程</a>
    
    <a href="/tags/Oracle/" style="font-size: 10.569242784523173px;" class="tag-cloud-link">Oracle</a>
    
    <a href="/tags/加密/" style="font-size: 16.201154597614796px;" class="tag-cloud-link">加密</a>
    
    <a href="/tags/ES6/" style="font-size: 18.932804569791646px;" class="tag-cloud-link">ES6</a>
    
    <a href="/tags/VueJs/" style="font-size: 10.63171532343554px;" class="tag-cloud-link">VueJs</a>
    
    <a href="/tags/代码段/" style="font-size: 12.612305287493054px;" class="tag-cloud-link">代码段</a>
    
    <a href="/tags/ASP-NET/" style="font-size: 12.905926365039615px;" class="tag-cloud-link">ASP.NET</a>
    
    <a href="/tags/XML/" style="font-size: 13.242580387228603px;" class="tag-cloud-link">XML</a>
    
    <a href="/tags/数据库/" style="font-size: 10.039644851136728px;" class="tag-cloud-link">数据库</a>
    
</div>
                <div class="friend-links tab-pane nav bs-sidenav fade" id="sidebar-friend-links">
    
    <li>
        <a href="http://www.baidu.com/" target="_blank">百度</a>
    </li>
    
</div>
                <div class="links tab-pane nav bs-sidenav fade" id="sidebar-links">
    
    <li>
        <a href="https://github.com/TaffyBlog" target="_blank">Github</a>
    </li>
    
    <li>
        <a href="https://www.zhihu.com/people/fei-ge-ge-2-8/activities" target="_blank">知乎</a>
    </li>
    
</div>
            </div>
        </div>
    </aside>
    
</aside>
                
            </div>
        </div>
    </div>
    <footer id="gal-footer">
    <div class="container">
        Copyright © 2018 Taffy Powered by <a href="https://hexo.io/" target="_blank">Hexo</a>.&nbsp;Theme by <a href="https://github.com/ZEROKISEKI" target="_blank">AONOSORA</a>
    </div>
</footer>

<!-- 回到顶端 -->
<div id="gal-gotop">
    <i class="fa fa-angle-up"></i>
</div>
<script src="/live2dw/lib/L2Dwidget.min.js?0c58a1486de42ac6cc1c59c7d98ae887"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","model":{"scale":1,"hHeadPos":0.5,"vHeadPos":0.618,"jsonPath":"/live2dw/assets/assets/hibiki.model.json"},"display":{"superSample":1.5,"width":150,"height":300,"position":"right","hOffset":0,"vOffset":0},"mobile":{"show":true,"scale":0.1},"react":{"opacityDefault":0.7,"opacityOnHover":0.2},"log":false,"tagMode":false});</script></body>
<script src="/js/activate-power-mode.js"></script>
<script>

    // 配置highslide
	hs.graphicsDir = '/js/highslide/graphics/'
    hs.outlineType = "rounded-white";
    hs.dimmingOpacity = 0.8;
    hs.outlineWhileAnimating = true;
    hs.showCredits = false;
    hs.captionEval = "this.thumb.alt";
    hs.numberPosition = "caption";
    hs.align = "center";
    hs.transitions = ["expand", "crossfade"];
    hs.lang.number = '共%2张图, 当前是第%1张';
    hs.addSlideshow({
      interval: 5000,
      repeat: true,
      useControls: true,
      fixedControls: "fit",
      overlayOptions: {
        opacity: 0.75,
        position: "bottom center",
        hideOnMouseOut: true
      }
    })

    // 初始化aos
    AOS.init({
      duration: 1000,
      delay: 0,
      easing: 'ease-out-back'
    });

</script>
<script>
	POWERMODE.colorful = 'true';    // make power mode colorful
	POWERMODE.shake = 'true';       // turn off shake
	// TODO 这里根据具体情况修改
	document.body.addEventListener('input', POWERMODE);
</script>
<script>
    window.slideConfig = {
      prefix: 'http://p9hqbt6wm.bkt.clouddn.com/background',
      ext: 'jpg',
      maxCount: '6'
    }
</script>
<script src="/js/hs.js"></script>
<script src="/js/blog.js"></script>

<script src="/js/oni.js"></script>



<script src="https://imsun.github.io/gitment/dist/gitment.browser.js"></script>
<script>
    if(window.commentConfig) {
      window.commentConfig.id = 'Sat Oct 14 2017 13:03:13 GMT+0800'
      window.commentConfig.owner = 'TaffyBlog'
      window.commentConfig.repo = 'taffyblogcomments'
      window.commentConfig.client_id = 'fd4a3c1821147e2a3dfe'
      window.commentConfig.client_secret = 'de6bc8eec3ca751089ca7cca196515cb6a4a7c24'
      window.commentConfig.redirect_uri = 'https://taffyblog.github.io'
    } else {
      window.commentConfig = {
      	id: 'Sat Oct 14 2017 13:03:13 GMT+0800',
        owner: 'TaffyBlog',
        repo: 'taffyblogcomments',
        client_id: 'fd4a3c1821147e2a3dfe',
        client_secret: 'de6bc8eec3ca751089ca7cca196515cb6a4a7c24',
        redirect_uri: 'https://taffyblog.github.io'
      }
    }
</script>
<script src="/js/comment/gitment.js"></script>

</html>